# -*- coding: utf-8 -*-
import scrapy
import re
import requests
from lxml import etree
from urllib import parse
from lpw.items import LpwItem


class LiepinSpider(scrapy.Spider):
    name = 'liepin'
    allowed_domains = ['liepin.com']
    start_urls = ['https://www.liepin.com/zhaopin/?init=-1&headckid=88f82049310e2b2b&flushckid=1&fromSearchBtn=2&ckid=3cd1a3ea18157175&degradeFlag=0&curPage=0&industryType=industry_01&industries=040&siTag=1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_fp_nvbar&d_ckId=7a9e5aece499e0122f9d104610e77c14&d_curPage=0&d_pageSize=40&d_headId=d3cda0518b689d406eab0b3b5efdf9be']
    #start_urls= ['https://www.liepin.com/zhaopin/?init=-1&headckid=88f82049310e2b2b&flushckid=1&fromSearchBtn=2&ckid=72f00258ecb17ab7&degradeFlag=0&curPage=0&industryType=industry_01&industries=040&siTag=1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_fp_nvbar&d_ckId=2baef21a856ebae808042c2cd0047dcc&d_curPage=0&d_pageSize=40&d_headId=d3cda0518b689d406eab0b3b5efdf9be']

    def parse(self, response):
        # with open('liepin.html','wb') as f:
        #     f.write(response.body)
        hrefs = response.xpath('//ul[@class="sojob-list"]//h3/a/@href').extract()
        print(hrefs)
        for href in hrefs:
            if 'a' in href:
                href = 'https://www.liepin.com'+href
                yield scrapy.Request(href,callback=self.parse_info_two)
            else:
                yield scrapy.Request(href,callback=self.parse_info)

        next = response.xpath('//div[@class="pagerbar"]/a[text()="下一页"]/@href').extract_first()
        next = 'https://www.liepin.com/' + next
        if next:
            yield scrapy.Request(next,callback=self.parse)
            print(next)


    def parse_info(self,response):
        title = response.xpath('//h1/text()').extract_first()
        # print(title)
        price = response.xpath('//p[@class="job-item-title"]/text()').extract_first()
        # print(price)
        weizhi = response.xpath('//p[@class="basic-infor"]/span/a/text()').extract_first()
        # print(weizhi)
        jingyan = response.xpath('//div[@class="job-qualifications"]/span[2]/text()').extract_first()
        # print(jingyan)
        xueli = response.xpath('//div[@class="job-qualifications"]/span[1]/text()').extract_first()
        # print(xueli)
        shijian = response.xpath('//p[@class="basic-infor"]/time/text()').extract_first()
        # print(shijian)
        info = response.xpath('//div[@class="job-item main-message job-description"]/div/text()').extract()
        info = ''.join(info)
        # print(info)
        wangzhan = '猎聘'
        # print(wangzhan)

        item = LpwItem()
        item['title'] = title
        item['price'] = price
        item['weizhi'] = weizhi
        item['jingyan'] = jingyan
        item['xueli'] = xueli
        item['shijian'] = shijian
        item['info'] = info
        item['wangzhan'] = wangzhan
        yield item

    def parse_info_two(self, response):
        title = response.xpath('//h1/text()').extract_first()
        # print(title)
        price = response.xpath('//p[@class="job-main-title"]/text()').extract_first()
        # print(price)
        weizhi = response.xpath('//p[@class="basic-infor"]/span/a/text()').extract_first()
        # print(weizhi)
        jingyan = response.xpath('//div[@class="resume clearfix"]/span[2]/text()').extract_first()
        # print(jingyan)
        xueli = response.xpath('//div[@class="resume clearfix"]/span[1]/text()').extract_first()
        # print(xueli)
        shijian = response.xpath('//p[@class="basic-infor"]/time/text()').extract_first()
        # print(shijian)
        info = response.xpath('//div[@class="job-item main-message job-description"]/div/text()').extract()
        info = ''.join(info)
        # print(info)
        wangzhan = '猎聘网'
        # print(wangzhan)
        item = LpwItem()
        item['title'] = title
        item['price'] = price
        item['weizhi'] = weizhi
        item['jingyan'] = jingyan
        item['xueli'] = xueli
        item['shijian'] = shijian
        item['info'] = info
        item['wangzhan'] = wangzhan
        yield item







